{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-09T11:47:16.292140Z",
     "start_time": "2025-10-09T11:47:16.287165Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\JNU\\Project\\Python\\2025FMa\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import logging\n",
    "import transformers\n",
    "import os\n",
    "os.environ['TRANSFORMERS_NO_ADVISORY_WARNINGS'] = '1'\n",
    "transformers.logging.set_verbosity_error()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b23612c66418a2af",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-09T11:47:19.090661Z",
     "start_time": "2025-10-09T11:47:19.083333Z"
    }
   },
   "outputs": [],
   "source": [
    "def predict_sentiment_batch(model, data, tokenizer, device, batch_size=16):\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "\n",
    "    predicted_class = []\n",
    "\n",
    "    for i in tqdm(range(0, len(data), batch_size), desc=\"批量预测\", unit=\"batch\"):\n",
    "        batch_texts = data[i:i + batch_size]\n",
    "\n",
    "        inputs = tokenizer(\n",
    "            batch_texts,\n",
    "            return_tensors=\"pt\",\n",
    "            truncation=True,\n",
    "            padding=True,\n",
    "            max_length=512,\n",
    "            add_special_tokens=True\n",
    "        )\n",
    "        inputs = {key: value.to(device) for key, value in inputs.items()}\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**inputs)\n",
    "\n",
    "        predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
    "        batch_preds = torch.argmax(predictions, dim=1).cpu().numpy()\n",
    "        predicted_class.extend(batch_preds.tolist())\n",
    "\n",
    "    return predicted_class\n",
    "\n",
    "\n",
    "def calculate_detailed_metrics(y_true, y_pred):\n",
    "    y_true = np.array(y_true)\n",
    "    y_pred = np.array(y_pred)\n",
    "\n",
    "    accuracy = accuracy_score(y_true, y_pred)\n",
    "    precision = precision_score(y_true, y_pred, average='binary', zero_division=0)\n",
    "    recall = recall_score(y_true, y_pred, average='binary', zero_division=0)\n",
    "    f1 = f1_score(y_true, y_pred, average='binary', zero_division=0)\n",
    "\n",
    "    total = len(y_true)\n",
    "\n",
    "    metrics = {\n",
    "        'accuracy': accuracy,\n",
    "        'precision': precision,\n",
    "        'recall': recall,\n",
    "        'f1_score': f1,\n",
    "        'sample_info': {\n",
    "            'total_samples': total,\n",
    "            'true_positive': np.sum((y_true == 1) & (y_pred == 1)),\n",
    "            'true_negative': np.sum((y_true == 0) & (y_pred == 0)),\n",
    "            'false_positive': np.sum((y_true == 0) & (y_pred == 1)),\n",
    "            'false_negative': np.sum((y_true == 1) & (y_pred == 0)),\n",
    "        }\n",
    "    }\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def main_optimized():\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(f\"使用设备: {device}\")\n",
    "\n",
    "    test_data_path = './data/dev.csv'\n",
    "    df = pd.read_csv(test_data_path)\n",
    "    df.dropna(inplace=True)\n",
    "    data = df['SentimentText'].tolist()\n",
    "    target = df['label'].astype(int).tolist()\n",
    "\n",
    "    print(f\"数据加载完成，共 {len(data)} 个样本\")\n",
    "\n",
    "    model_paths = ['bert-base-chinese', './review_model', './review_classifier_model']\n",
    "\n",
    "    for model_path in model_paths:\n",
    "        print(f\"\\n{'=' * 60}\")\n",
    "        print(f\"评估模型: {model_path}\")\n",
    "        print(f\"{'=' * 60}\")\n",
    "\n",
    "        try:\n",
    "            model = AutoModelForSequenceClassification.from_pretrained(model_path)\n",
    "            tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "\n",
    "            batch_size = 32 if device.type == \"cuda\" else 8\n",
    "            print(f\"使用batch大小: {batch_size}\")\n",
    "\n",
    "            predictions = predict_sentiment_batch(model, data, tokenizer, device, batch_size)\n",
    "            metrics = calculate_detailed_metrics(target, predictions)\n",
    "\n",
    "            print(f\"\\n评估结果:\")\n",
    "            print(f\"   准确率 (Accuracy):  {metrics['accuracy']:.4f}\")\n",
    "            print(f\"   精确率 (Precision): {metrics['precision']:.4f}\")\n",
    "            print(f\"   召回率 (Recall):    {metrics['recall']:.4f}\")\n",
    "            print(f\"   F1分数:            {metrics['f1_score']:.4f}\")\n",
    "\n",
    "            print(f\"\\n样本统计:\")\n",
    "            info = metrics['sample_info']\n",
    "            print(f\"   总样本数: {info['total_samples']}\")\n",
    "            print(f\"   真正例 (TP): {info['true_positive']}\")\n",
    "            print(f\"   真反例 (TN): {info['true_negative']}\")\n",
    "            print(f\"   假正例 (FP): {info['false_positive']}\")\n",
    "            print(f\"   假反例 (FN): {info['false_negative']}\")\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"处理模型 {model_path} 时出错: {e}\")\n",
    "            continue\n",
    "    if device.type == \"cuda\":\n",
    "        torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "93cbd37d811f3d5a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-09T11:48:09.799924Z",
     "start_time": "2025-10-09T11:47:23.781163Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "使用设备: cuda\n",
      "数据加载完成，共 4988 个样本\n",
      "\n",
      "============================================================\n",
      "评估模型: bert-base-chinese\n",
      "============================================================\n",
      "使用batch大小: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "批量预测: 100%|██████████| 156/156 [00:21<00:00,  7.10batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "评估结果:\n",
      "   准确率 (Accuracy):  0.4896\n",
      "   精确率 (Precision): 0.4895\n",
      "   召回率 (Recall):    0.9996\n",
      "   F1分数:            0.6572\n",
      "\n",
      "样本统计:\n",
      "   总样本数: 4988\n",
      "   真正例 (TP): 2440\n",
      "   真反例 (TN): 2\n",
      "   假正例 (FP): 2545\n",
      "   假反例 (FN): 1\n",
      "\n",
      "============================================================\n",
      "评估模型: ./review_model\n",
      "============================================================\n",
      "使用batch大小: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "批量预测: 100%|██████████| 156/156 [00:22<00:00,  7.00batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "评估结果:\n",
      "   准确率 (Accuracy):  0.9092\n",
      "   精确率 (Precision): 0.8998\n",
      "   召回率 (Recall):    0.9164\n",
      "   F1分数:            0.9081\n",
      "\n",
      "样本统计:\n",
      "   总样本数: 4988\n",
      "   真正例 (TP): 2237\n",
      "   真反例 (TN): 2298\n",
      "   假正例 (FP): 249\n",
      "   假反例 (FN): 204\n",
      "\n",
      "============================================================\n",
      "评估模型: ./review_classifier_model\n",
      "============================================================\n",
      "使用batch大小: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "批量预测: 100%|██████████| 156/156 [00:21<00:00,  7.14batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "评估结果:\n",
      "   准确率 (Accuracy):  0.8605\n",
      "   精确率 (Precision): 0.8528\n",
      "   召回率 (Recall):    0.8640\n",
      "   F1分数:            0.8584\n",
      "\n",
      "样本统计:\n",
      "   总样本数: 4988\n",
      "   真正例 (TP): 2109\n",
      "   真反例 (TN): 2183\n",
      "   假正例 (FP): 364\n",
      "   假反例 (FN): 332\n"
     ]
    }
   ],
   "source": [
    "main_optimized()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
